{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9e71a8c7",
   "metadata": {},
   "source": [
    "在双模型的基础上,进一步缓解自举.\n",
    "\n",
    "在计算target时,使用原模型计算动作,使用延迟模型计算target,而不是直接取max."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAATyklEQVR4nO3df2yT950H8LedxA758TgkNHZzxAd3q0pz/OgWIDzrSZuGR9ZF01gzaduhLqs4qjIHlWZCWqRCBVcpFdOpKxsNf1SD/tOxS3VsakTLRQ4N2tWQkpK7kEJuk5gSUWzzY3mcBGI79uf+6PJsLoHG+eGvjd8v6anq5/ux/Xm+id88P2LbIiICIiIFrKobIKLcxQAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJllAXQ4cOHsWLFChQWFqKurg69vb2qWiEiRZQE0G9+8xu0tLTgpZdewkcffYR169ahvr4eoVBIRTtEpIhFxZtR6+rqsGHDBvzyl78EACQSCVRXV2PXrl346U9/mu52iEiR/HQ/YTQaRV9fH1pbW811VqsVHo8Hfr9/xvtEIhFEIhHzdiKRwK1bt1BRUQGLxbLoPRNRakQEY2NjqKqqgtV67wOttAfQjRs3EI/H4XQ6k9Y7nU5cvnx5xvu0tbVh//796WiPiBbQyMgIli9ffs/xtAfQXLS2tqKlpcW8bRgG3G43RkZGoGmaws6IaCbhcBjV1dUoLS29b13aA2jZsmXIy8tDMBhMWh8MBuFyuWa8j91uh91uv2u9pmkMIKIM9nmnSNJ+Fcxms6G2thY+n89cl0gk4PP5oOt6utshIoWUHIK1tLSgqakJ69evx8aNG/Hzn/8cExMTeOaZZ1S0Q0SKKAmg733ve7h+/Tr27duHQCCAxx9/HO+9995dJ6aJ6MGm5O+A5iscDsPhcMAwDJ4DIspAs32N8r1gRKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUiblADpz5gy+9a1voaqqChaLBb/97W+TxkUE+/btw8MPP4wlS5bA4/HgD3/4Q1LNrVu3sG3bNmiahrKyMmzfvh3j4+Pz2hAiyj4pB9DExATWrVuHw4cPzzh+8OBBHDp0CEeOHMG5c+dQXFyM+vp6TE5OmjXbtm3D4OAgurq60NnZiTNnzuDZZ5+d+1YQUXaSeQAgJ06cMG8nEglxuVzys5/9zFw3Ojoqdrtdfv3rX4uIyMcffywA5MMPPzRr3n33XbFYLHL16tVZPa9hGAJADMOYT/tEtEhm+xpd0HNAV65cQSAQgMfjMdc5HA7U1dXB7/cDAPx+P8rKyrB+/XqzxuPxwGq14ty5czM+biQSQTgcTlqIKPstaAAFAgEAgNPpTFrvdDrNsUAggMrKyqTx/Px8lJeXmzWf1dbWBofDYS7V1dUL2TYRKZIVV8FaW1thGIa5jIyMqG6JiBbAggaQy+UCAASDwaT1wWDQHHO5XAiFQknjU1NTuHXrllnzWXa7HZqmJS1ElP0WNIBWrlwJl8sFn89nrguHwzh37hx0XQcA6LqO0dFR9PX1mTXd3d1IJBKoq6tbyHaIKMPlp3qH8fFx/PGPfzRvX7lyBf39/SgvL4fb7cbu3bvx8ssv45FHHsHKlSuxd+9eVFVVYevWrQCAxx57DN/4xjewY8cOHDlyBLFYDM3Nzfj+97+PqqqqBdswIsoCqV5eO336tAC4a2lqahKRTy/F7927V5xOp9jtdtm8ebMMDQ0lPcbNmzflBz/4gZSUlIimafLMM8/I2NjYgl/iIyI1ZvsatYiIKMy/OQmHw3A4HDAMg+eDiDLQbF+jWXEVjIgeTAwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISJmUv5aHaCHd+fM13L4xbN7OsxfDUf1PsFgsCruidGEAkVLG8ABGzr5t3i6qqIb2d4/BkpensCtKFx6CUQbKum+KojliAFFGEfO7LikXMIAoszB/cgoDiDIMEyiXMIAowwiy8NvCaY4YQJRZmD05JaUAamtrw4YNG1BaWorKykps3boVQ0NDSTWTk5Pwer2oqKhASUkJGhsbEQwGk2qGh4fR0NCAoqIiVFZWYs+ePZiampr/1lDWEwjAPaCckVIA9fT0wOv14uzZs+jq6kIsFsOWLVswMTFh1rzwwgt455130NHRgZ6eHnzyySd46qmnzPF4PI6GhgZEo1F88MEHePPNN3Hs2DHs27dv4baKspeY/6FcIPMQCoUEgPT09IiIyOjoqBQUFEhHR4dZc+nSJQEgfr9fREROnjwpVqtVAoGAWdPe3i6apkkkEpnV8xqGIQDEMIz5tE8Z4Fr/Kek9ssNc/vf4XolFbqtui+Zptq/ReZ0DMgwDAFBeXg4A6OvrQywWg8fjMWtWrVoFt9sNv98PAPD7/VizZg2cTqdZU19fj3A4jMHBwRmfJxKJIBwOJy30oOIhWC6ZcwAlEgns3r0bTzzxBFavXg0ACAQCsNlsKCsrS6p1Op0IBAJmzd+Gz/T49NhM2tra4HA4zKW6unqubVOGEx6C5ZQ5B5DX68XFixdx/PjxhexnRq2trTAMw1xGRkYW/TlJFWH+5JA5vRm1ubkZnZ2dOHPmDJYvX26ud7lciEajGB0dTdoLCgaDcLlcZk1vb2/S401fJZuu+Sy73Q673T6XVinbcA8op6S0ByQiaG5uxokTJ9Dd3Y2VK1cmjdfW1qKgoAA+n89cNzQ0hOHhYei6DgDQdR0DAwMIhUJmTVdXFzRNQ01NzXy2hR4IwlNAOSSlPSCv14u33noLv/vd71BaWmqes3E4HFiyZAkcDge2b9+OlpYWlJeXQ9M07Nq1C7quY9OmTQCALVu2oKamBk8//TQOHjyIQCCAF198EV6vl3s59JeTQEygXJFSALW3twMAvvrVryatP3r0KH70ox8BAF599VVYrVY0NjYiEomgvr4er7/+ulmbl5eHzs5O7Ny5E7quo7i4GE1NTThw4MD8toQeCJ8egTGAcoVFJPt+2uFwGA6HA4ZhQNM01e3QPAT+57+SPpDMVlqBmq2tKCjizzWbzfY1yveCUWaR6c8EolzAAKIMwz9EzCUMICJShgFEGUWEe0C5hAFEGUZ4DiiHMIBIrc9+/xf3gHIKA4iUKnQ4YbH+9TvApibHEbvDTzvIFQwgUspizUvaCxJJAJJQ2BGlEwOI1LLwVzCX8adPSvE74HMbA4gUs/xloVzEACK1uAeU0xhApBQPwXIbA4jUYgDlNAYQKWXh+Z+cxgAitSyMoFzGACK1LBYehuUwBhApxf2f3MYAIrW495PTGECkFgMopzGASCkeguU2BhCpxT2gnMYAIrUsfC9YLmMAkVJ8K0ZuS+mbUYlSJSIYHx9HPB6fcTw6Np78GdACjI+NY8o+OmO9xWJBaWkprFb+2/kgYADRoorH4/jud7+LwcHBGccfLi/Cv//rP6PQ9umvokgC/7JtG/7v6uiM9ZWVlfD5fFi6dOlitUxpxACiRSUiCIVCuHr16ozjlqiG8JQDlyObEJd8rCy8gOvXr+Pq1dCM9fF4HIkEP7L1QZHSfmx7ezvWrl0LTdOgaRp0Xce7775rjk9OTsLr9aKiogIlJSVobGxEMBhMeozh4WE0NDSgqKgIlZWV2LNnD6amphZmayjrROJ29Ic3IxD9B1yP/T0ujHlwO+5Q3RalSUoBtHz5crzyyivo6+vD+fPn8bWvfQ3f/va3zd3rF154Ae+88w46OjrQ09ODTz75BE899ZR5/3g8joaGBkSjUXzwwQd48803cezYMezbt29ht4qyxlQiD3cSpZi+EhZJFCEqhWqbovSReVq6dKm88cYbMjo6KgUFBdLR0WGOXbp0SQCI3+8XEZGTJ0+K1WqVQCBg1rS3t4umaRKJRGb9nIZhCAAxDGO+7dMii0aj8vjjjwuAGZeHljrktX87KgdePif7Xz4nh9r+Ux5dUX3PepfLJTdu3FC9WfQ5ZvsanfM5oHg8jo6ODkxMTEDXdfT19SEWi8Hj8Zg1q1atgtvtht/vx6ZNm+D3+7FmzRo4nU6zpr6+Hjt37sTg4CC++MUvptTD5cuXUVJSMtdNoDSYmprC5OTkPcdv37mN/z59CLcS/4iE5KHSdgW3/nzzvo83NDSEsrKyReiWFsr4+Pis6lIOoIGBAei6jsnJSZSUlODEiROoqalBf38/bDbbXb8YTqcTgUAAABAIBJLCZ3p8euxeIpEIIpGIeTsc/vSL6wzD4PmjDBePx+95CR4AJiZj+I/uCwAuzOrxEomE+fOnzDUxMTGrupQD6NFHH0V/fz8Mw8Dbb7+NpqYm9PT0pNxgKtra2rB///671tfV1UHTtEV9bpqfWCyG4uLiBXs8m82GDRs2oKKiYsEekxbebP+RSPmvuWw2G77whS+gtrYWbW1tWLduHV577TW4XC5Eo1GMjo4m1QeDQbhcLgCAy+W666rY9O3pmpm0trbCMAxzGRkZSbVtIspA8/5z0kQigUgkgtraWhQUFMDn85ljQ0NDGB4ehq7rAABd1zEwMIBQ6K9/49HV1QVN01BTU3PP57Db7eal/+mFiLJfSodgra2tePLJJ+F2uzE2Noa33noL77//Pk6dOgWHw4Ht27ejpaUF5eXl0DQNu3btgq7r2LRpEwBgy5YtqKmpwdNPP42DBw8iEAjgxRdfhNfrhd1uX5QNJKLMlVIAhUIh/PCHP8S1a9fgcDiwdu1anDp1Cl//+tcBAK+++iqsVisaGxsRiURQX1+P119/3bx/Xl4eOjs7sXPnTui6juLiYjQ1NeHAgQMLu1WUUUpKShZsr7W0tJRvYH2AWEREPr8ss4TDYTgcDhiGwcOxDCciCAaDiMViC/J4VqsVLpcLeXl5C/J4tDhm+xrle8FoUVkslvteYKDcxs80ICJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMrkq25gLkQEABAOhxV3QkQzmX5tTr9W7yUrA+jmzZsAgOrqasWdENH9jI2NweFw3HM8KwOovLwcADA8PHzfjaNk4XAY1dXVGBkZgaZpqtvJCpyzuRERjI2Noaqq6r51WRlAVuunp64cDgd/KeZA0zTOW4o4Z6mbzc4BT0ITkTIMICJSJisDyG6346WXXoLdblfdSlbhvKWOc7a4LPJ518mIiBZJVu4BEdGDgQFERMowgIhIGQYQESmTlQF0+PBhrFixAoWFhairq0Nvb6/qlpRpa2vDhg0bUFpaisrKSmzduhVDQ0NJNZOTk/B6vaioqEBJSQkaGxsRDAaTaoaHh9HQ0ICioiJUVlZiz549mJqaSuemKPPKK6/AYrFg9+7d5jrOWZpIljl+/LjYbDb51a9+JYODg7Jjxw4pKyuTYDCoujUl6uvr5ejRo3Lx4kXp7++Xb37zm+J2u2V8fNysee6556S6ulp8Pp+cP39eNm3aJF/+8pfN8ampKVm9erV4PB65cOGCnDx5UpYtWyatra0qNimtent7ZcWKFbJ27Vp5/vnnzfWcs/TIugDauHGjeL1e83Y8Hpeqqippa2tT2FXmCIVCAkB6enpERGR0dFQKCgqko6PDrLl06ZIAEL/fLyIiJ0+eFKvVKoFAwKxpb28XTdMkEomkdwPSaGxsTB555BHp6uqSr3zlK2YAcc7SJ6sOwaLRKPr6+uDxeMx1VqsVHo8Hfr9fYWeZwzAMAH99w25fXx9isVjSnK1atQput9ucM7/fjzVr1sDpdJo19fX1CIfDGBwcTGP36eX1etHQ0JA0NwDnLJ2y6s2oN27cQDweT/qhA4DT6cTly5cVdZU5EokEdu/ejSeeeAKrV68GAAQCAdhsNpSVlSXVOp1OBAIBs2amOZ0eexAdP34cH330ET788MO7xjhn6ZNVAUT35/V6cfHiRfz+979X3UpGGxkZwfPPP4+uri4UFhaqbienZdUh2LJly5CXl3fX1YhgMAiXy6Woq8zQ3NyMzs5OnD59GsuXLzfXu1wuRKNRjI6OJtX/7Zy5XK4Z53R67EHT19eHUCiEL33pS8jPz0d+fj56enpw6NAh5Ofnw+l0cs7SJKsCyGazoba2Fj6fz1yXSCTg8/mg67rCztQRETQ3N+PEiRPo7u7GypUrk8Zra2tRUFCQNGdDQ0MYHh4250zXdQwMDCAUCpk1XV1d0DQNNTU16dmQNNq8eTMGBgbQ399vLuvXr8e2bdvM/+ecpYnqs+CpOn78uNjtdjl27Jh8/PHH8uyzz0pZWVnS1YhcsnPnTnE4HPL+++/LtWvXzOX27dtmzXPPPSdut1u6u7vl/Pnzouu66Lpujk9fUt6yZYv09/fLe++9Jw899FBOXVL+26tgIpyzdMm6ABIR+cUvfiFut1tsNpts3LhRzp49q7olZQDMuBw9etSsuXPnjvz4xz+WpUuXSlFRkXznO9+Ra9euJT3On/70J3nyySdlyZIlsmzZMvnJT34isVgszVujzmcDiHOWHvw4DiJSJqvOARHRg4UBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKfP/5TGe93yhVroAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ecfbe912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       " ),\n",
       " Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型,评估状态下每个动作的价值\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "#延迟更新的模型,用于计算target\n",
    "model_delay = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "#复制参数\n",
    "model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "model, model_delay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\appDir\\python3.10\\lib\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  if not isinstance(terminated, (bool, np.bool8)):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-992.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "        if random.random() < 0.1:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bfcfd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_3876\\2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(207,\n",
       " (array([ 0.02789014, -0.02109897,  0.04514677,  0.00323187], dtype=float32),\n",
       "  1,\n",
       "  1.0,\n",
       "  array([ 0.02746816,  0.17334741,  0.0452114 , -0.27487186], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 408 -990.85\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(100):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                #使用原模型计算动作,使用延迟模型计算target,进一步缓解自举\n",
    "                next_action = model(next_state).argmax(dim=1, keepdim=True)\n",
    "                target = model_delay(next_state).gather(dim=1,\n",
    "                                                        index=next_action)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        #复制参数\n",
    "        if (epoch + 1) % 5 == 0:\n",
    "            model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVsElEQVR4nO3dfWxT570H8K8dv+T12M2b3ZR4MJUWMl66BUgM0zbRjAyyaqzZ1TqhLqtQe8scBs2Eukgtve02paJS23WjcCttUGlimajGKqLSLje0Qb0YKOlybwgkaye65JLYCaE5dkJiO/Zz/6hy2gMJiZPgB4fvRzpSz/M8tn/naf2t/ZxzHIMQQoCISAKj7AKI6PbFACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImmkBdDevXuxcOFCpKamoqSkBGfOnJFVChFJIiWA/vznP6OmpgbPPPMMPvzwQ6xcuRLl5eXo6+uTUQ4RSWKQcTNqSUkJVq9ejd/97ncAgFgshsLCQmzfvh2/+MUvEl0OEUliSvQLhsNhtLS0oLa2VmszGo0oKyuD1+ud8DGhUAihUEjbj8ViuHLlCnJycmAwGG56zUQUHyEEgsEgCgoKYDRO/kUr4QF0+fJlRKNROBwOXbvD4UBHR8eEj6mrq8Ozzz6biPKIaA51d3djwYIFk/YnPIBmora2FjU1Ndq+qqpwuVzo7u6GoigSKyOiiQQCARQWFiIrK+uG4xIeQLm5uUhJSYHf79e1+/1+OJ3OCR9jtVphtVqva1cUhQFEdAubaokk4WfBLBYLiouL0dTUpLXFYjE0NTXB7XYnuhwikkjKV7CamhpUVVVh1apVWLNmDV5++WUMDw/jkUcekVEOEUkiJYB++MMfor+/H7t374bP58N9992Ht99++7qFaSKa36RcBzRbgUAANpsNqqpyDYjoFjTd9yjvBSMiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0cQfQiRMn8MADD6CgoAAGgwF//etfdf1CCOzevRt33nkn0tLSUFZWho8++kg35sqVK9iyZQsURYHdbsfWrVsxNDQ0qwMhouQTdwANDw9j5cqV2Lt374T9e/bswSuvvIL9+/fj9OnTyMjIQHl5OUZHR7UxW7ZsQXt7OxobG9HQ0IATJ07gsccem/lREFFyErMAQBw5ckTbj8Viwul0ihdeeEFrGxwcFFarVfzpT38SQghx/vx5AUB88MEH2phjx44Jg8EgLl26NK3XVVVVABCqqs6mfCK6Sab7Hp3TNaCLFy/C5/OhrKxMa7PZbCgpKYHX6wUAeL1e2O12rFq1ShtTVlYGo9GI06dPT/i8oVAIgUBAtxFR8pvTAPL5fAAAh8Oha3c4HFqfz+dDfn6+rt9kMiE7O1sbc626ujrYbDZtKywsnMuyiUiSpDgLVltbC1VVta27u1t2SUQ0B+Y0gJxOJwDA7/fr2v1+v9bndDrR19en6x8bG8OVK1e0MdeyWq1QFEW3EVHym9MAWrRoEZxOJ5qamrS2QCCA06dPw+12AwDcbjcGBwfR0tKijTl+/DhisRhKSkrmshwiusWZ4n3A0NAQPv74Y23/4sWLaG1tRXZ2NlwuF3bu3Ilf/epXWLx4MRYtWoSnn34aBQUF2Lx5MwBg6dKl+M53voNHH30U+/fvRyQSQXV1NR566CEUFBTM2YERURKI9/Tau+++KwBct1VVVQkhPjsV//TTTwuHwyGsVqu4//77RWdnp+45BgYGxI9+9CORmZkpFEURjzzyiAgGg3N+io+I5Jjue9QghBAS829GAoEAbDYbVFXlehDRLWi679GkOAtGRPMTA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNLE/XfBiOgzkasB9He8j7Q77kSq3QFzuh0pljQYDAbZpSUNBhDRDAV7/4GelqMQsRhSLKkwZ9hxd9m/Iy2bf2BzuhhARDMghMDIpz0QsSgAIBoegdFkgdFslVxZcuEaENFMiBiG/Bd1TeZ0GywZdjn1JCkGENEMRCOjCAX6dG0Z+V8GDHxLxYOzRTQDI5/2IhQc+EKLARm5hdLqSVYMIKIZCAcHABHT9o1mKzIcX+YZsDjFFUB1dXVYvXo1srKykJ+fj82bN6Ozs1M3ZnR0FB6PBzk5OcjMzERlZSX8fr9uTFdXFyoqKpCeno78/Hzs2rULY2Njsz8aogQQQiBwqUPXlmJJhTk1S1JFySuuAGpubobH48GpU6fQ2NiISCSCDRs2YHh4WBvzxBNP4OjRozh8+DCam5vR09ODBx98UOuPRqOoqKhAOBzGyZMn8frrr+PgwYPYvXv33B0V0U0kopFrvn4BGXkLYUrNkFRR8jIIIcRMH9zf34/8/Hw0NzfjG9/4BlRVRV5eHg4dOoQf/OAHAICOjg4sXboUXq8XpaWlOHbsGL773e+ip6cHDocDALB//348+eST6O/vh8VimfJ1A4EAbDYbVFWFoigzLZ9oRsLDg2h/4zmMjQ5pbfnL1uNL6x6SWNWtZbrv0VmtAamqCgDIzs4GALS0tCASiaCsrEwbs2TJErhcLni9XgCA1+vF8uXLtfABgPLycgQCAbS3t0/4OqFQCIFAQLcRyXJ1oBvR8KiuzbbgK5KqSW4zDqBYLIadO3di3bp1WLZsGQDA5/PBYrHAbrfrxjocDvh8Pm3MF8NnvH+8byJ1dXWw2WzaVljIsw0khxACIbUPIvb5mqUpNRNWW57EqpLXjAPI4/Hg3LlzqK+vn8t6JlRbWwtVVbWtu7v7pr8m0cQEgr0f6VpMqZmwZGZLqie5zehWjOrqajQ0NODEiRNYsGCB1u50OhEOhzE4OKj7FOT3++F0OrUxZ86c0T3f+Fmy8THXslqtsFp5iTvJJ6JjCAX6dW0Z+QthNPKuppmI6xOQEALV1dU4cuQIjh8/jkWLFun6i4uLYTab0dTUpLV1dnaiq6sLbrcbAOB2u9HW1oa+vs+vIm1sbISiKCgqKprNsRDddKHgAELBy7q2VJsT4PU/MxJXbHs8Hhw6dAhvvvkmsrKytDUbm82GtLQ02Gw2bN26FTU1NcjOzoaiKNi+fTvcbjdKS0sBABs2bEBRUREefvhh7NmzBz6fD0899RQ8Hg8/5dAtLxToRzTy+QK0IcWETOfdvABxhuIKoH379gEAvvWtb+naDxw4gJ/85CcAgJdeeglGoxGVlZUIhUIoLy/Hq6++qo1NSUlBQ0MDtm3bBrfbjYyMDFRVVeG5556b3ZEQ3WSfXYB4AfjClSsp5jSkKrkSq0pus7oOSBZeB0QyxKJj+Od/vYbBT1q1tqyCe3HPph0wpnAN6IsSch0Q0e0kFglhuP8TXZtVyYPBmCKnoHmAAUQ0TaGhAcSuuQBRKbhXUjXzAwOIaBqEEBi5cumaBWgzLJnZXICeBQYQ0TQFLl3Q7ZvTspCWfZekauYHBhDRdIjYdXfAp+e6YDRNffM0TY4BRDQN4eFPMfppr64t1ebgAvQsMYCIpiEyEtTfAW8wIuvOxVz/mSUGENEUhBAI9nTq7oA3GFNg5QWIs8YAIpqKENfdgJpmd/IO+DnAACKaQiwawZDvn7o2U1oWjCbeuzhbDCCiKYSHP0X4qqprs7mWS6pmfmEAEU0hpPoRDX3+hxcMKSak2Z1cgJ4DDCCiKQz3/0u3n2JOQ1r2gklGUzwYQEQ3IEQMVy/rfwLYquQixZIqqaL5hQFEdANjoasYHdT/sYS07Lt4BfQcYQAR3cDY1QDCw4O6toy8L3H9Z44wgIhuIND7D8TGQtq+wZiC9FyXxIrmFwYQ0SSEEAhfcwOqJSsHqUq+pIrmHwYQ0SRELIpAT4euzZymcAF6DjGAiCYRDV3F2MiQrk25aylg4NtmrnAmiSYxqvoRHv708waDgb+AOMcYQESTGOq7CIiYtp9isiLrzsUSK5p/GEBEExBCYGTg/3RtlqwcmFIzJVU0PzGAiCYQjYzi6jUBlGp3IMWSJqmi+Yl/TY1uSyMjIwiFQpP2R0dUhIeu6BvT86Gq6oTjMzIyYDab57LE2wIDiG5LL7zwAl577bVJ+9d+5S7UbF4Bo7bgLPCzJ/8D//vJzyYcX19fj69//es3odL5jQFEt6VAIIBLly5N2m/+SgE+GimFOpaLAuvHSA21ovXCRfReGZpw/Ojo6ITtdGNxrQHt27cPK1asgKIoUBQFbrcbx44d0/pHR0fh8XiQk5ODzMxMVFZWwu/3656jq6sLFRUVSE9PR35+Pnbt2oWxsbFrX4pIGoPBgDsW/hsujqzAlchdOD+0Dm1+JwYCI7JLm3fiCqAFCxbg+eefR0tLC86ePYv169fje9/7Htrb2wEATzzxBI4ePYrDhw+jubkZPT09ePDBB7XHR6NRVFRUIBwO4+TJk3j99ddx8OBB7N69e26PimgW0lPNMKc5AXz29SsGE/678yoiY1G5hc1DcQXQAw88gE2bNmHx4sW455578Otf/xqZmZk4deoUVFXF73//e7z44otYv349iouLceDAAZw8eRKnTp0CAPztb3/D+fPn8cc//hH33XcfNm7ciF/+8pfYu3cvwuHwTTlAong57sjASkcPDCIMQMBqCCCktkPILmwemvEaUDQaxeHDhzE8PAy3242WlhZEIhGUlZVpY5YsWQKXywWv14vS0lJ4vV4sX74cDodDG1NeXo5t27ahvb0dX/3qV+OqoaOjA5mZvC6D4jcwMDBpX5dfxZ7/fBG5zkbk5X8ZqwqH8I+P/+eGz9fV1YXz58/PdZlJa2ho4rWya8UdQG1tbXC73RgdHUVmZiaOHDmCoqIitLa2wmKxwG6368Y7HA74fJ/9oJPP59OFz3j/eN9kQqGQ7pRpIBAAAKiqyvUjmpEbnYIfi8Zw4V99wL+aYDA04U2DAdHYjT//DA0NYXBwcI6rTF7Dw8NTD8IMAujee+9Fa2srVFXFG2+8gaqqKjQ3N8ddYDzq6urw7LPPXtdeUlICRVFu6mvT/PSXv/xlWuOEAKJi6i9fRUVFWLt27WzLmjfGPyRMJe4roS0WC+6++24UFxejrq4OK1euxG9+8xs4nU6Ew+Hr/i/g9/vhdDoBAE6n87qzYuP742MmUltbC1VVta27u3vSsUSUPGZ9K0YsFkMoFEJxcTHMZjOampq0vs7OTnR1dcHtdgMA3G432tra0NfXp41pbGyEoigoKiqa9DWsVqt26n98I6LkF9dXsNraWmzcuBEulwvBYBCHDh3Ce++9h3feeQc2mw1bt25FTU0NsrOzoSgKtm/fDrfbjdLSUgDAhg0bUFRUhIcffhh79uyBz+fDU089BY/HA6uVf2WS6HYTVwD19fXhxz/+MXp7e2Gz2bBixQq88847+Pa3vw0AeOmll2A0GlFZWYlQKITy8nK8+uqr2uNTUlLQ0NCAbdu2we12IyMjA1VVVXjuuefm9qiIpjD+qXouGAwGmEy8qWAmDEJMY4XtFhMIBGCz2aCqKr+O0YwMDg4iGAzO2fPl5eUhNZU/1Tpuuu9Rxjbdlux2+3WXjFDi8feAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQm2QXMhBACABAIBCRXQkQTGX9vjr9XJ5OUATQwMAAAKCwslFwJEd1IMBiEzWabtD8pAyg7OxsA0NXVdcODI71AIIDCwkJ0d3dDURTZ5SQFztnMCCEQDAZRUFBww3FJGUBG42dLVzabjf9RzICiKJy3OHHO4jedDwdchCYiaRhARCRNUgaQ1WrFM888A6vVKruUpMJ5ix/n7OYyiKnOkxER3SRJ+QmIiOYHBhARScMAIiJpGEBEJE1SBtDevXuxcOFCpKamoqSkBGfOnJFdkjR1dXVYvXo1srKykJ+fj82bN6Ozs1M3ZnR0FB6PBzk5OcjMzERlZSX8fr9uTFdXFyoqKpCeno78/Hzs2rULY2NjiTwUaZ5//nkYDAbs3LlTa+OcJYhIMvX19cJisYg//OEPor29XTz66KPCbrcLv98vuzQpysvLxYEDB8S5c+dEa2ur2LRpk3C5XGJoaEgb8/jjj4vCwkLR1NQkzp49K0pLS8XatWu1/rGxMbFs2TJRVlYm/v73v4u33npL5ObmitraWhmHlFBnzpwRCxcuFCtWrBA7duzQ2jlniZF0AbRmzRrh8Xi0/Wg0KgoKCkRdXZ3Eqm4dfX19AoBobm4WQggxODgozGazOHz4sDbmwoULAoDwer1CCCHeeustYTQahc/n08bs27dPKIoiQqFQYg8ggYLBoFi8eLFobGwU3/zmN7UA4pwlTlJ9BQuHw2hpaUFZWZnWZjQaUVZWBq/XK7GyW4eqqgA+v2G3paUFkUhEN2dLliyBy+XS5szr9WL58uVwOBzamPLycgQCAbS3tyew+sTyeDyoqKjQzQ3AOUukpLoZ9fLly4hGo7p/6QDgcDjQ0dEhqapbRywWw86dO7Fu3TosW7YMAODz+WCxWGC323VjHQ4HfD6fNmaiOR3vm4/q6+vx4Ycf4oMPPriuj3OWOEkVQHRjHo8H586dw/vvvy+7lFtad3c3duzYgcbGRqSmpsou57aWVF/BcnNzkZKSct3ZCL/fD6fTKamqW0N1dTUaGhrw7rvvYsGCBVq70+lEOBzG4OCgbvwX58zpdE44p+N9801LSwv6+vrwta99DSaTCSaTCc3NzXjllVdgMpngcDg4ZwmSVAFksVhQXFyMpqYmrS0Wi6GpqQlut1tiZfIIIVBdXY0jR47g+PHjWLRoka6/uLgYZrNZN2ednZ3o6urS5sztdqOtrQ19fX3amMbGRiiKgqKiosQcSALdf//9aGtrQ2trq7atWrUKW7Zs0f6Zc5YgslfB41VfXy+sVqs4ePCgOH/+vHjssceE3W7XnY24nWzbtk3YbDbx3nvvid7eXm27evWqNubxxx8XLpdLHD9+XJw9e1a43W7hdru1/vFTyhs2bBCtra3i7bffFnl5ebfVKeUvngUTgnOWKEkXQEII8dvf/la4XC5hsVjEmjVrxKlTp2SXJA2ACbcDBw5oY0ZGRsRPf/pTcccdd4j09HTx/e9/X/T29uqe55NPPhEbN24UaWlpIjc3V/z85z8XkUgkwUcjz7UBxDlLDP4cBxFJk1RrQEQ0vzCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaf4fhvMOk9g3a30AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-882.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
